{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Elastic Net Regression\n",
    "\n",
    "This function shows how to use TensorFlow to solve elastic net regression. \n",
    "$y = Ax + b$\n",
    "\n",
    "We will use the iris data, specifically:\n",
    "```\n",
    "#  y = Sepal Length\n",
    "#  x = Pedal Length, Petal Width, Sepal Width\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn import datasets\n",
    "from tensorflow.python.framework import ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up for TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ops.reset_default_graph()\n",
    "\n",
    "# Create graph\n",
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load the data\n",
    "# iris.data = [(Sepal Length, Sepal Width, Petal Length, Petal Width)]\n",
    "iris = datasets.load_iris()\n",
    "x_vals = np.array([[x[1], x[2], x[3]] for x in iris.data])\n",
    "y_vals = np.array([y[0] for y in iris.data])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make results reproducible\n",
    "seed = 13\n",
    "np.random.seed(seed)\n",
    "tf.set_random_seed(seed)\n",
    "\n",
    "# Declare batch size\n",
    "batch_size = 50\n",
    "\n",
    "# Initialize placeholders\n",
    "x_data = tf.placeholder(shape=[None, 3], dtype=tf.float32)\n",
    "y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)\n",
    "\n",
    "# Create variables for linear regression\n",
    "A = tf.Variable(tf.random_normal(shape=[3,1]))\n",
    "b = tf.Variable(tf.random_normal(shape=[1,1]))\n",
    "\n",
    "# Declare model operations\n",
    "model_output = tf.add(tf.matmul(x_data, A), b)\n",
    "\n",
    "# Declare the elastic net loss function\n",
    "elastic_param1 = tf.constant(1.)\n",
    "elastic_param2 = tf.constant(1.)\n",
    "l1_a_loss = tf.reduce_mean(tf.abs(A))\n",
    "l2_a_loss = tf.reduce_mean(tf.square(A))\n",
    "e1_term = tf.multiply(elastic_param1, l1_a_loss)\n",
    "e2_term = tf.multiply(elastic_param2, l2_a_loss)\n",
    "loss = tf.expand_dims(tf.add(tf.add(tf.reduce_mean(tf.square(y_target - model_output)), e1_term), e2_term), 0)\n",
    "\n",
    "# Declare optimizer\n",
    "my_opt = tf.train.GradientDescentOptimizer(0.001)\n",
    "train_step = my_opt.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step #250 A = [[ 1.26014626]\n",
      " [ 0.4016138 ]\n",
      " [ 0.40159121]] b = [[-0.14889474]]\n",
      "Loss = [ 1.59188581]\n",
      "Step #500 A = [[ 1.17897248]\n",
      " [ 0.46715766]\n",
      " [ 0.29896322]] b = [[-0.0677181]]\n",
      "Loss = [ 1.46314824]\n",
      "Step #750 A = [[ 1.13416564]\n",
      " [ 0.51899707]\n",
      " [ 0.21090424]] b = [[ 0.01904622]]\n",
      "Loss = [ 1.37157845]\n",
      "Step #1000 A = [[ 1.09745109]\n",
      " [ 0.54604095]\n",
      " [ 0.13102381]] b = [[ 0.10402215]]\n",
      "Loss = [ 1.27774763]\n"
     ]
    }
   ],
   "source": [
    "# Initialize variables\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)\n",
    "\n",
    "# Training loop\n",
    "loss_vec = []\n",
    "for i in range(1000):\n",
    "    rand_index = np.random.choice(len(x_vals), size=batch_size)\n",
    "    rand_x = x_vals[rand_index]\n",
    "    rand_y = np.transpose([y_vals[rand_index]])\n",
    "    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y})\n",
    "    temp_loss = sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y})\n",
    "    loss_vec.append(temp_loss[0])\n",
    "    if (i+1)%250==0:\n",
    "        print('Step #' + str(i+1) + ' A = ' + str(sess.run(A)) + ' b = ' + str(sess.run(b)))\n",
    "        print('Loss = ' + str(temp_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract model results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the optimal coefficients\n",
    "[[sw_coef], [pl_coef], [pw_ceof]] = sess.run(A)\n",
    "[y_intercept] = sess.run(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+cXHV97/HXe2dnh2x+EJA1D0iQgI3egheCRsRib6no\nhSvUYLXeeP2RUFpqH6jV1rbQ3iK9mlt7bwUv/qjFggavCiiK6MMrAlq5UBsMXEQSwAQIkBjIGshP\nkt3Z2c/945zZnKxnZn+Q2ZnseT8fj3nsme/59Tkzu/Pe7/fMnFFEYGZmNlpXuwswM7PO5IAwM7Nc\nDggzM8vlgDAzs1wOCDMzy+WAMDOzXA4IM0PS5yT9TbvrsM7igLApI2mjpDe0u45WkLRI0vWS+iXt\nlLRe0qckLWh3baNJWiHprmxbRLw3Ij7arpqsMzkgzCZAUndO268Bq4FfAKdGxBzgDOBR4HXtrs9s\nshwQ1hEk/aGkDZKelXSLpGPSdkm6UtLW9D/zn0l6RTrvTZLWSdolabOkDzfY9gpJd0v6tKQdkh6W\ndFZm/uGSrpG0Jd3OxySVRq17paRtwOU5u7gcuDsi/jQiNgFExNaI+GREXJ/Zz3mS7pe0XdK/Sjo5\nM2+jpA9LeiCt8QZJh01g3b+U9ACwR1K3pEskPZo+NuskvSVd9teBzwGvlbRb0va0/YuSPjbW85HO\nC0nvTXtJ2yV9RpLGfpbtkBMRvvk2JTdgI/CGnPbXA78EXglUgE8Bd6bzzgbuBeYCAn4dODqdtwX4\nzXT6COCVDfa7AhgCPgSUgf8M7ACOTOd/E/gnYCbwYuAe4I9Grft+oBuYkbP9p4EVYxz7qcBW4DVA\nCViePh6VzGNzD3AMcCTwEPDeCax7P3BsvT7g99JtdaXHuyfzuK0A7hpV3xeBj431fKTzA/hO+py8\nBOgHzmn375dvB//mHoR1gncC10bEfRExAFxK8h/uQqAKzAb+HaCIeCgitqTrVYETJc2JiOci4r4m\n+9gKfDIiqhFxA/AIcK6kecCbgA9GxJ6I2ApcCSzLrPuLiPhURAxFxN6cbR9FEhIASHpf+p/1bkmf\nT5svAv4pIlZHRC0iVgEDwOmZ7VwVEb+IiGeBbwOLJ7juU/X6IuJr6baG0+NdD5zW5PHJavZ81H08\nIrZHxJPADzO12jTigLBOcAzwRP1OROwGtgHzI+IHwKeBzwBbJV0taU666FtJXtyfkPQjSa9tso/N\nEZG9MuUT6X6PI+lVbElf1LeT9CZenFn2qTHq3wYcnan/0xExF/hkum3S/fxZfR/pfo5Na6h7OjP9\nPDBrAuseUKOk92SGpLYDryAJsvFo+HyMo1abRhwQ1gl+QfIiCICkmcCLgM0AEXFVRLwKOBF4GfDn\naftPImIpyYv5zcCNTfYxf9Q4+UvS/T5F8t/4URExN73NiYiTMsuOdcnjO4DfHWOZp4CVmX3MjYje\niPjqGOuNd92RGiUdB3weeB/wojSsHiQZohvP8TR9Pqw4HBA21cqSDsvcuoGvAhdIWiypAvx3YHVE\nbJT0akmvkVQmGUffBwxL6pH0TkmHR0QV2AkMN9nvi4EPSCpL+j2ScxnfTYervg98QtIcSV2SXirp\ntyZwTJcDvynpCknzASQdle6j7vPAe9NjkaSZks6VNHsc25/oujNJQqA/reUCkh5E3TPAAkk9DdZv\n+HyMo1abRhwQNtW+C+zN3C6PiNuBvwFuIjnx/FL2nwOYQ/IC+RzJsMc24H+m894NbJS0E3gvydh5\nI6uBRSQnX1cCb4uIbem89wA9wLp0P18nM2Q0loj4OckJ5AXATyXtAu4m+U/8b9Jl1gB/SDJc9hyw\ngeRk8Xi2P6F1I2Id8AngxyRh8O/Teup+AKwFnpb0y5z1mz0fViA6cFjWbPqRtAL4g4iY0s8kmB3q\n3IMwM7NcDggzM8vlISYzM8vlHoSZmeU6pC/sddRRR8XChQvbXYaZ2SHl3nvv/WVE9I213CEdEAsX\nLmTNmjXtLsPM7JAi6Ymxl/IQk5mZNeCAMDOzXA4IMzPL5YAwM7NcDggzM8vlgDAzs1wOCDMzy1XI\ngNi0aROXXXYZP//5z9tdiplZxypkQGzZsoWPfvSjrF+/vt2lmJl1rEIGRFdXctjDw82+gMzMrNgK\nHRC1Wq3NlZiZda5CBkSpVALcgzAza6aQAeEhJjOzsRU6IDzEZGbWWCEDwkNMZmZjK2RAuAdhZja2\nQgaEexBmZmMrZED4JLWZ2dgKHRAeYjIza6yQAeEhJjOzsRUyIDzEZGY2tkIHhIeYzMwaa1lASDpM\n0j2SfippraS/TduPlHSbpPXpzyMy61wqaYOkRySd3araPMRkZja2VvYgBoDXR8QpwGLgHEmnA5cA\nd0TEIuCO9D6STgSWAScB5wCflVRqRWEeYjIzG1vLAiISu9O75fQWwFJgVdq+Cjg/nV4KXB8RAxHx\nOLABOK0VtXmIycxsbC09ByGpJOl+YCtwW0SsBuZFxJZ0kaeBeen0fOCpzOqb0rbR27xI0hpJa/r7\n+ydVl4eYzMzG1tKAiIhaRCwGFgCnSXrFqPlB0quYyDavjoglEbGkr69vUnV5iMnMbGxT8i6miNgO\n/JDk3MIzko4GSH9uTRfbDBybWW1B2nbQeYjJzGxsrXwXU5+kuen0DOCNwMPALcDydLHlwLfS6VuA\nZZIqko4HFgH3tKI2DzGZmY2tu4XbPhpYlb4TqQu4MSK+I+nHwI2SLgSeAN4OEBFrJd0IrAOGgIsj\noiX/4rsHYWY2tpYFREQ8AJya074NOKvBOiuBla2qqc49CDOzsRXyk9SSAAeEmVkzhQwISIaZPMRk\nZtZYYQOiVCq5B2Fm1kRhA6Krq8sBYWbWRKEDwkNMZmaNFTYgPMRkZtZcYQPCQ0xmZs0VOiA8xGRm\n1lhhA8JDTGZmzRU2IDzEZGbWXGEDolQqMTQ01O4yzMw6VmEDolwuU61W212GmVnHckCYmVmuwgZE\nT08Pg4OD7S7DzKxjFTYg3IMwM2uusAHhHoSZWXOFDQj3IMzMmitsQLgHYWbWXGEDwj0IM7PmChsQ\n7kGYmTVX2IBwD8LMrLmWBYSkYyX9UNI6SWsl/UnafrmkzZLuT29vyqxzqaQNkh6RdHaragP3IMzM\nxtLdwm0PAX8WEfdJmg3cK+m2dN6VEfEP2YUlnQgsA04CjgFul/SyiGjJNbndgzAza65lPYiI2BIR\n96XTu4CHgPlNVlkKXB8RAxHxOLABOK1V9bkHYWbW3JScg5C0EDgVWJ02vV/SA5KulXRE2jYfeCqz\n2iZyAkXSRZLWSFrT398/6ZrcgzAza67lASFpFnAT8MGI2An8I3ACsBjYAnxiItuLiKsjYklELOnr\n65t0Xd3d3b7ct5lZEy0NCEllknD4ckR8AyAinomIWkQMA59n/zDSZuDYzOoL0raWKJVK/spRM7Mm\nWvkuJgHXAA9FxBWZ9qMzi70FeDCdvgVYJqki6XhgEXBPq+pzQJiZNdfKdzGdAbwb+Jmk+9O2vwLe\nIWkxEMBG4I8AImKtpBuBdSTvgLq4Ve9gAgeEmdlYWhYQEXEXoJxZ322yzkpgZatqynJAmJk1V9hP\nUjsgzMyaK2xA+F1MZmbNFTYgSqUSw8PDRES7SzEz60iFDgiA4eHhNldiZtaZCh8QPg9hZpbPAeGA\nMDPL5YBwQJiZ5XJAOCDMzHIVNiC6u5PPCPqtrmZm+QobEO5BmJk154BwQJiZ5XJAOCDMzHI5IBwQ\nZma5HBAOCDOzXA4IB4SZWa7CBoTf5mpm1lxhA8I9CDOz5hwQDggzs1wOCAeEmVmuwgZEuVwGoFqt\ntrkSM7POVNiAmDVrFgC7d+9ucyVmZp2pZQEh6VhJP5S0TtJaSX+Sth8p6TZJ69OfR2TWuVTSBkmP\nSDq7VbUBzJkzB4CdO3e2cjdmZoesVvYghoA/i4gTgdOBiyWdCFwC3BERi4A70vuk85YBJwHnAJ+V\nVGpVcbNnzwZg165drdqFmdkhrWUBERFbIuK+dHoX8BAwH1gKrEoXWwWcn04vBa6PiIGIeBzYAJzW\nqvrcgzAza25KzkFIWgicCqwG5kXElnTW08C8dHo+8FRmtU1pW0u4B2Fm1lzLA0LSLOAm4IMRccC/\n6xERQExwexdJWiNpTX9//6TrqlQqdHd3OyDMzBpoaUBIKpOEw5cj4htp8zOSjk7nHw1sTds3A8dm\nVl+Qth0gIq6OiCURsaSvr++F1EZvby/PP//8pLdhZjadtfJdTAKuAR6KiCsys24BlqfTy4FvZdqX\nSapIOh5YBNzTqvoAZs6cyZ49e1q5CzOzQ1Z3C7d9BvBu4GeS7k/b/gr4OHCjpAuBJ4C3A0TEWkk3\nAutI3gF1cUS09GPO7kGYmTXWsoCIiLsANZh9VoN1VgIrW1XTaO5BmJk1VthPUoN7EGZmzRQ6INyD\nMDNrrNAB4R6EmVljhQ4I9yDMzBordEC4B2Fm1lihA8I9CDOzxgodEO5BmJk1VuiAmDlzJoODgwwN\nDbW7FDOzjjOugJD0UkmVdPpMSR+QNLe1pbVeb28vgHsRZmY5xtuDuAmoSfo14GqSi+p9pWVVTZGZ\nM2cC+DyEmVmO8QbEcEQMAW8BPhURfw4c3bqypkalUgFgYGCgzZWYmXWe8QZEVdI7SK6++p20rdya\nkqZOqZR8o2mt1tJrApqZHZLGGxAXAK8FVkbE4+nluL/UurKmhgPCzKyxcV3NNSLWAR8AkHQEMDsi\n/r6VhU2F7u7k8P0uJjOzXzXedzH9i6Q5ko4E7gM+L+mKsdbrdO5BmJk1Nt4hpsPT75P+XeC6iHgN\n8IbWlTU1HBBmZo2NNyC60++Pfjv7T1If8hwQZmaNjTcg/htwK/BoRPxE0gnA+taVNTUcEGZmjY33\nJPXXgK9l7j8GvLVVRU0VB4SZWWPjPUm9QNI3JW1NbzdJWtDq4lrNAWFm1th4h5i+ANwCHJPevp22\nHdIcEGZmjY03IPoi4gsRMZTevgj0tbCuKeGAMDNrbLwBsU3SuySV0tu7gG3NVpB0bToc9WCm7XJJ\nmyXdn97elJl3qaQNkh6RdPbkDmdi/EE5M7PGxhsQv0/yFtengS3A24AVY6zzReCcnPYrI2Jxevsu\ngKQTgWXASek6n5VUGmdtk+YehJlZY+MKiIh4IiLeHBF9EfHiiDifMd7FFBF3As+Os46lwPURMRAR\njwMbgNPGue6kOSDMzBp7Id8o96eTXO/9kh5Ih6COSNvmA09lltmUtv0KSRdJWiNpTX9//yRLSDgg\nzMwaeyEBoUms84/ACcBikqGqT0x0AxFxdUQsiYglfX0v7Dy5A8LMrLEXEhAx4RUinomIWkQMA59n\n/zDSZpJvqatbkLa1lAPCzKyxpgEhaZeknTm3XSSfh5iQ9HpOdW8B6u9wugVYJqmSftfEIuCeiW5/\nohwQZmaNNb3URkTMnuyGJX0VOBM4StIm4CPAmZIWk/Q+NgJ/lO5nraQbgXXAEHBxRLT8VdsBYWbW\n2LiuxTQZEfGOnOZrmiy/EljZqnryOCDMzBp7IecgDnn+oJyZWWOFDgj3IMzMGnNA4IAwM8vjgABu\nv/32NldiZtZ5Ch0Qc+fOBWDr1q1trsTMrPMUOiB6enp4/etfz/DwcLtLMTPrOIUOCIBKpcLAwEC7\nyzAz6zgOCAeEmVmuwgdET08Pg4OD7S7DzKzjFD4g3IMwM8vngHBAmJnlckA4IMzMchU+IHwOwsws\nX+EDwj0IM7N8DohKhWq16g/LmZmNUviA6OnpAfAwk5nZKIUPiEqlAjggzMxGc0CkAeHzEGZmB3JA\nOCDMzHIVPiB8DsLMLF/hA8I9CDOzfC0LCEnXStoq6cFM25GSbpO0Pv15RGbepZI2SHpE0tmtqms0\nB4SZWb5W9iC+CJwzqu0S4I6IWATckd5H0onAMuCkdJ3PSiq1sLYRDggzs3wtC4iIuBN4dlTzUmBV\nOr0KOD/Tfn1EDETE48AG4LRW1ZblcxBmZvmm+hzEvIjYkk4/DcxLp+cDT2WW25S2/QpJF0laI2lN\nf3//Cy7IPQgzs3xtO0kdEQHEJNa7OiKWRMSSvr6+F1yHA8LMLN9UB8Qzko4GSH9uTds3A8dmlluQ\ntrVcfYjJAWFmdqCpDohbgOXp9HLgW5n2ZZIqko4HFgH3TEVBM2bMAOBLX/rSVOzOzOyQ0d2qDUv6\nKnAmcJSkTcBHgI8DN0q6EHgCeDtARKyVdCOwDhgCLo6IWqtqy5o1axYAN99881TszszskNGygIiI\ndzSYdVaD5VcCK1tVTyP1gDAzswMV/pPU2YDwd0KYme1X+IAolfZ/Hq9arbaxEjOzzlL4gMjyh+XM\nzPZzQGQ4IMzM9nNAAH/3d38HOCDMzLIcEMC8eckVPxwQZmb7OSDwBfvMzPI4IHBAmJnlcUDg6zGZ\nmeVxQOAehJlZHgcE7kGYmeVxQAC9vb0A7N27t82VmJl1DgcEMGfOHAB27tzZ5krMzDqHAwIHhJlZ\nHgcEDggzszwOCPZf8tsBYWa2nwOC5JLfs2bNckCYmWU4IFJz5sxxQJiZZTggUnPmzGH79u3tLsPM\nrGO07DupDzUPP/wwDz/8MI899hgnnHBCu8sxM2s79yBGWb9+fbtLMDPrCG3pQUjaCOwCasBQRCyR\ndCRwA7AQ2Ai8PSKem+rayuXyVO/SzKwjtbMH8dsRsTgilqT3LwHuiIhFwB3p/Slz4YUXArBnz56p\n3K2ZWcfqpCGmpcCqdHoVcP5U7vzDH/4wALt3757K3ZqZdax2BUQAt0u6V9JFadu8iNiSTj8NzMtb\nUdJFktZIWtPf33/QCqp/WM4BYWaWaNe7mF4XEZslvRi4TdLD2ZkREZIib8WIuBq4GmDJkiW5y0xG\nPSB27dp1sDZpZnZIa0sPIiI2pz+3At8ETgOekXQ0QPpz61TWNHv2bCTx3HNTfl7czKwjTXlASJop\naXZ9GviPwIPALcDydLHlwLemsq5SqURfXx/PPPPMVO7WzKxjtWOIaR7wTUn1/X8lIr4n6SfAjZIu\nBJ4A3j7lhc2b54AwM0tNeUBExGPAKTnt24CzprqerGOOOYaNGze2swQzs47RSW9zbbszzjiDBx54\ngB07drS7FDOztnNAZBx//PEAbN06pefHzcw6kgMi48gjjwTg2WefbXMlZmbt54DIeNGLXgQ4IMzM\nwAFxgHoPYtu2bW2uxMys/RwQGR5iMjPbzwGRMXfuXCQ5IMzMcEAcoFQqcdhhh/HjH/+43aWYmbWd\nA2KUvXv38v3vf59qtdruUszM2soBMUr9nUxPPvlkmysxM2svB8QoN910EwAbNmxocyVmZu3lgBjl\nlFNOQRKrV69udylmZm3lgBhl7ty5nHLKKdx5553tLsXMrK0cEDnOPPNM7r77bp5//vl2l2Jm1jYO\niBxvfOMb2bdvH2vWrGl3KWZmbeOAyLFo0SIAHn/88TZXYmbWPg6IHMcddxwAK1asYPfu3W2uxsys\nPRwQOXp6ejjvvPMAfLLazArLAdHADTfcAMC5557rb5gzs0JyQDTQ29s7Mr1ixQqGh4fbWI2Z2dRz\nQDTx3HPPcdlll3HzzTdz7rnnMjQ01O6SzMymjAOiiblz53L55ZfzO7/zO3zve9/jggsu4NZbb3Vv\nwswKQRHR7hoOIOkc4H8BJeCfI+LjjZZdsmRJTMVnFXbt2sX73vc+rrvuOiAZfvrABz7AS1/6Unp6\nepg5cyZz586lUqnQ09Mz8rP+2FYqFQ477LCR+5JGbl1dXSPTAwMD9PT00NXVNbJcs5/jWUYSw8PD\n7Nq1i9mzZ1OpVKhWqwwODlIqlX5lX/V6u7q6GBgYYGBggFqtRrlcRtLIOhExsk6pVCIiqNVqDA0N\nMTQ0RERw+OGHU61W6e7uplarsXfvXiqVCgC1Wo0ZM2aMrFvfVkQQESMhXCqVDqitkVqtRrVaHdl+\ndlv1xzkiqFar9PT0jMwfHh4eeQ5GGxwcHDnu8coei1mnknRvRCwZc7lOCghJJeDnwBuBTcBPgHdE\nxLq85acqICD5w1+zZg0f+tCH2LdvHz/96U895HQQSCIiKJfL1Gq1g9I7qwdZrVY7ICCGh4dH9tXV\n1cXQ0BC1Wm1knXqY1kNhcHDwgDDPLpfdTz3kIoKhoSG6u7tHgrK+THd3N8CYx9coXJq114+hHnT1\nMKzvq1wuA4wENUD27z77j8quXbvo7e2lVqtRq9Xo7u4eCdHsDZIA7e3tpVqtMjw8zPDw8MhzWL/1\n9vbS09NDuVymWq1O6G+m/phm/ykZHh6mVCqxb98+Zs+efcC+y+XyAY9/uVymu7t7ZDv1Y87+U1Y/\n5nK5PLI+wIwZM9i3b9/I414/5rGek9Ft41lmsts677zzuOqqq35lvfEYb0B0T2rrrXMasCEiHgOQ\ndD2wFMgNiKkkiVe/+tXcddddAOzevZvHHnuMHTt2MGPGDPbs2TPyn/nAwACDg4Ps27ePSqUycr/+\nYjj6Vn/hqlQqDA4OHvDL3Ohns3l5P3t7exkYGGDfvn2Uy2V6enpGXkCz2xweHmZwcJChoSF6enpG\nTtbX2+t/qPW2+gtU/YWw/kc5PDzMjh07qFQqDA0NjcyrVqt0dXVRKpXYs2fPyP6r1erIi22211B/\nwcn7A83q6uqiXC6zd+/ekd5M/cWh3hPo6uqiUqmwe/duhoeH6e7uplKpHPCC1tXVRbVapVarMWfO\nHAYGBg54HEdPZ0Ot/uIkie7u7pFQyIZFNmBGa/TPWrP2+gto/fnI9oqyvaZ6rfVls49n9sV9xowZ\n1Gq1kRfRgYGBkcDMBkA9aJ9//nkqlcrIi3j9ua1Pb9++faSG+u/GRHpY2f1mg75UKrF3717K5fLI\n70v9d6j+3NcDKRuAo//mIoIZM2YcUGNEjIRD/XhGPwdj3T/YbXnLvPzlL89/0A6iTguI+cBTmfub\ngNdkF5B0EXARwEte8pKpq2yUWbNmcfLJJ7dt/2ZmrXbInaSOiKsjYklELOnr62t3OWZm01anBcRm\n4NjM/QVpm5mZTbFOC4ifAIskHS+pB1gG3NLmmszMCqmjzkFExJCk9wG3krzN9dqIWNvmsszMCqmj\nAgIgIr4LfLfddZiZFV2nDTGZmVmHcECYmVkuB4SZmeXqqEttTJSkfuCJF7CJo4BfHqRyDgVFO17w\nMReFj3lijouIMT9IdkgHxAslac14rkcyXRTteMHHXBQ+5tbwEJOZmeVyQJiZWa6iB8TV7S5gihXt\neMHHXBQ+5hYo9DkIMzNrrOg9CDMza8ABYWZmuQoZEJLOkfSIpA2SLml3PQeLpGMl/VDSOklrJf1J\n2n6kpNskrU9/HpFZ59L0cXhE0tntq37yJJUk/T9J30nvT+vjBZA0V9LXJT0s6SFJr53Oxy3pQ+nv\n9IOSvirpsOl4vJKulbRV0oOZtgkfp6RXSfpZOu8qTeRr/LLyvgJzOt9IrhL7KHAC0AP8FDix3XUd\npGM7GnhlOj2b5Pu9TwT+B3BJ2n4J8Pfp9Inp8VeA49PHpdTu45jEcf8p8BXgO+n9aX286bGsAv4g\nne4B5k7X4yb5psnHgRnp/RuBFdPxeIH/ALwSeDDTNuHjBO4BTgcE/B/gP02mniL2IEa+9zoiBoH6\n914f8iJiS0Tcl07vAh4i+eNaSvKCQvrz/HR6KXB9RAxExOPABpLH55AhaQFwLvDPmeZpe7wAkg4n\neSG5BiAiBiNiO9P7uLuBGZK6gV7gF0zD442IO4FnRzVP6DglHQ3MiYh/iyQtrsusMyFFDIi8772e\n36ZaWkbSQuBUYDUwLyK2pLOeBual09Phsfgk8BfAcKZtOh8vJP8t9gNfSIfW/lnSTKbpcUfEZuAf\ngCeBLcCOiPg+0/R4c0z0OOen06PbJ6yIATHtSZoF3AR8MCJ2Zuel/1FMi/c2SzoP2BoR9zZaZjod\nb0Y3yTDEP0bEqcAekqGHEdPpuNMx96UkwXgMMFPSu7LLTKfjbWaqj7OIATGtv/daUpkkHL4cEd9I\nm59Ju52kP7em7Yf6Y3EG8GZJG0mGCl8v6X8zfY+3bhOwKSJWp/e/ThIY0/W43wA8HhH9EVEFvgH8\nBtP3eEeb6HFuTqdHt09YEQNi2n7vdfpOhWuAhyLiisysW4Dl6fRy4FuZ9mWSKpKOBxaRnNw6JETE\npRGxICIWkjyPP4iIdzFNj7cuIp4GnpL08rTpLGAd0/e4nwROl9Sb/o6fRXJ+bboe72gTOs50OGqn\npNPTx+s9mXUmpt1n7dtxA95E8g6fR4G/bnc9B/G4XkfS/XwAuD+9vQl4EXAHsB64HTgys85fp4/D\nI0zynQ6dcAPOZP+7mIpwvIuBNelzfTNwxHQ+buBvgYeBB4EvkbxzZ9odL/BVkvMsVZKe4oWTOU5g\nSfpYPQp8mvSqGRO9+VIbZmaWq4hDTGZmNg4OCDMzy+WAMDOzXA4IMzPL5YAwM7NcDggrFEnzJH1F\n0mOS7pX0Y0lvaVMtZ0r6jcz990p6TztqMcvT3e4CzKZK+qGhm4FVEfFf0rbjgDe3cJ/dETHUYPaZ\nwG7gXwEi4nOtqsNsMvw5CCsMSWcBl0XEb+XMKwEfJ3nRrgCfiYh/knQmcDnwS+AVwL3AuyIiJL0K\nuAKYlc5fERFbJP0LyYcUX0fywaefA/+V5LLc24B3AjOAfwNqJBfeez/JJ4R3R8Q/SFoMfI7kyqWP\nAr8fEc+l214N/DbJJb4vjIj/e/AeJbP9PMRkRXIScF+DeReSXCX01cCrgT9ML18AyVVxP0hy/f0T\ngDPSa159CnhbRLwKuBZYmdleT0QsiYhPAHcBp0dyYb3rgb+IiI0kAXBlRCzOeZG/DvjLiDgZ+Bnw\nkcy87og4La3pI5i1iIeYrLAkfYbkv/xB4AngZElvS2cfTnJtm0GS69tsSte5H1gIbCfpUdyWfllX\nieQSCXU3ZKYXADekF1rrIfnym2Z1HQ7MjYgfpU2rgK9lFqlfhPHetBazlnBAWJGsBd5avxMRF0s6\niuSaRk8C74+IW7MrpENMA5mmGsnfjYC1EfHaBvvak5n+FHBFRNySGbJ6Ier11GsxawkPMVmR/AA4\nTNIfZ9rcp1NvAAAAyElEQVR605+3An+cDh0h6WXpl/A08gjQJ+m16fJlSSc1WPZw9l9ueXmmfRfJ\nV8MeICJ2AM9J+s206d3Aj0YvZ9Zq/u/DCiM9sXw+cKWkvyA5ObwH+EuSIZyFwH3pu536afI1jREx\nmA5HXZUOCXWTfLvd2pzFLwe+Juk5kpCqn9v4NvB1SUtJTlJnLQc+J6kXeAy4YOJHbPbC+F1MZmaW\ny0NMZmaWywFhZma5HBBmZpbLAWFmZrkcEGZmlssBYWZmuRwQZmaW6/8D0hEGY1aeiuMAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116a93dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot loss over time\n",
    "plt.plot(loss_vec, 'k-')\n",
    "plt.title('Loss per Generation')\n",
    "plt.xlabel('Generation')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tf-cpu]",
   "language": "python",
   "name": "conda-env-tf-cpu-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
